{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a7cc0eab",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\anaconda3\\envs\\mashup\\lib\\site-packages\\torchaudio\\extension\\extension.py:13: UserWarning: torchaudio C++ extension is not available.\n",
      "  warnings.warn('torchaudio C++ extension is not available.')\n",
      "D:\\anaconda3\\envs\\mashup\\lib\\site-packages\\torchaudio\\backend\\utils.py:89: UserWarning: No audio backend is available.\n",
      "  warnings.warn('No audio backend is available.')\n"
     ]
    }
   ],
   "source": [
    "from transformers import pipeline\n",
    "import pandas as pd\n",
    "import nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "d750d8e2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[]\n"
     ]
    }
   ],
   "source": [
    "from transformers import AutoTokenizer, AutoModelForTokenClassification\n",
    "from transformers import pipeline\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"dslim/bert-base-NER\")\n",
    "model = AutoModelForTokenClassification.from_pretrained(\"D:/transformers_models/bert-base-NER\")\n",
    "\n",
    "nlp = pipeline(\"ner\", model=model, tokenizer=tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "ba31961f-a0b4-453c-a81f-29fccc4d5e87",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[{'word': 'Move', 'score': 0.8183884620666504, 'entity': 'B-MISC', 'index': 16, 'start': 60, 'end': 64}, {'word': '##able', 'score': 0.969035267829895, 'entity': 'I-MISC', 'index': 17, 'start': 64, 'end': 68}, {'word': 'Weather', 'score': 0.9304888248443604, 'entity': 'I-MISC', 'index': 18, 'start': 69, 'end': 76}, {'word': 'Move', 'score': 0.8764826059341431, 'entity': 'B-MISC', 'index': 25, 'start': 99, 'end': 103}, {'word': '##able', 'score': 0.9252719879150391, 'entity': 'I-MISC', 'index': 26, 'start': 103, 'end': 107}, {'word': 'Tide', 'score': 0.9125745296478271, 'entity': 'I-MISC', 'index': 27, 'start': 108, 'end': 112}, {'word': '##s', 'score': 0.9478297829627991, 'entity': 'I-MISC', 'index': 28, 'start': 112, 'end': 113}, {'word': 'T', 'score': 0.9617962837219238, 'entity': 'B-MISC', 'index': 34, 'start': 131, 'end': 132}, {'word': '##rop', 'score': 0.6572962403297424, 'entity': 'I-MISC', 'index': 35, 'start': 132, 'end': 135}, {'word': '##o', 'score': 0.9516863226890564, 'entity': 'I-MISC', 'index': 36, 'start': 135, 'end': 136}, {'word': 'Google', 'score': 0.8686999082565308, 'entity': 'I-MISC', 'index': 38, 'start': 137, 'end': 143}, {'word': 'La', 'score': 0.9780943393707275, 'entity': 'I-MISC', 'index': 39, 'start': 144, 'end': 146}, {'word': '##ti', 'score': 0.9758349061012268, 'entity': 'I-MISC', 'index': 40, 'start': 146, 'end': 148}, {'word': '##tude', 'score': 0.970452070236206, 'entity': 'I-MISC', 'index': 41, 'start': 148, 'end': 152}, {'word': 'Wikipedia', 'score': 0.47928303480148315, 'entity': 'I-MISC', 'index': 43, 'start': 153, 'end': 162}, {'word': 'Wikipedia', 'score': 0.6468074321746826, 'entity': 'B-ORG', 'index': 50, 'start': 184, 'end': 193}, {'word': 'National', 'score': 0.8995717167854309, 'entity': 'B-MISC', 'index': 57, 'start': 231, 'end': 239}, {'word': 'Regis', 'score': 0.9760364294052124, 'entity': 'I-MISC', 'index': 58, 'start': 240, 'end': 245}, {'word': '##try', 'score': 0.9827428460121155, 'entity': 'I-MISC', 'index': 59, 'start': 245, 'end': 248}, {'word': 'of', 'score': 0.9881213307380676, 'entity': 'I-MISC', 'index': 60, 'start': 249, 'end': 251}, {'word': 'Historic', 'score': 0.9891709685325623, 'entity': 'I-MISC', 'index': 61, 'start': 252, 'end': 260}, {'word': 'Places', 'score': 0.9802908897399902, 'entity': 'I-MISC', 'index': 62, 'start': 261, 'end': 267}, {'word': 'U', 'score': 0.9993715882301331, 'entity': 'B-LOC', 'index': 65, 'start': 270, 'end': 271}, {'word': '.', 'score': 0.9951863884925842, 'entity': 'I-LOC', 'index': 66, 'start': 271, 'end': 272}, {'word': 'S', 'score': 0.9988307356834412, 'entity': 'I-LOC', 'index': 67, 'start': 272, 'end': 273}]\n"
     ]
    }
   ],
   "source": [
    "example = 'This is the 3rd in the \"moveable\" series, that started with Moveable Weather, and then moved on to Moveable Tides. This time, this Tropo/Google Latitude/Wikipedia mashup reads to from Wikipedia pages about places registered on the National Registry of Historic Places. (U.S only). You can surf virtually, or actually visit the locations with your GPS-enabled phone.'\n",
    "\n",
    "ner_results = nlp(example)\n",
    "print(ner_results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "a0134c41",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<transformers.pipelines.token_classification.TokenClassificationPipeline at 0x15fbbcbd348>"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "4f0e3c7f",
   "metadata": {},
   "outputs": [],
   "source": [
    "mashup = pd.read_pickle('mashup_df.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "d8080605",
   "metadata": {},
   "outputs": [],
   "source": [
    "mashup['Related APIs'] = mashup['Related APIs'].map(lambda x: [y.strip() for y in x.split(',')] if x is not None else [])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "eb75be90",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mashup_name</th>\n",
       "      <th>Related APIs</th>\n",
       "      <th>Categories</th>\n",
       "      <th>URL</th>\n",
       "      <th>Company</th>\n",
       "      <th>Mashup/App Type</th>\n",
       "      <th>Description</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Twitwhere</td>\n",
       "      <td>[Twitter, Google Earth, Google App Engine]</td>\n",
       "      <td>Mapping, Social, Visualizations</td>\n",
       "      <td>http://twitwhere.appspot.com</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>This Service is a mashup service using twitter...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Infinitube</td>\n",
       "      <td>[YouTube]</td>\n",
       "      <td>Video</td>\n",
       "      <td>http://infinitube.net</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Youtube-powered infinitely-playing video viewe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Hotelplease</td>\n",
       "      <td>[Google Maps]</td>\n",
       "      <td>Mapping, Travel</td>\n",
       "      <td>http://hosting.gmodules.com/ig/gadgets/file/11...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Search the prices on 43,000 hotels worldwide. ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Donations</td>\n",
       "      <td>[PayPal]</td>\n",
       "      <td>Social, Charity, Charity</td>\n",
       "      <td>http://www.qmpeople.com/helpme/</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Thanks to PayPal, users can donate money to he...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>PetStew.com</td>\n",
       "      <td>[Flickr, Amazon Product Advertising, Google Ma...</td>\n",
       "      <td>Pets, Pets, Classifieds, Video, Photos, eCommerce</td>\n",
       "      <td>http://www.petstew.com/index.html</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>A pet site providing classifieds, adoptions, p...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   mashup_name                                       Related APIs  \\\n",
       "0    Twitwhere         [Twitter, Google Earth, Google App Engine]   \n",
       "1   Infinitube                                          [YouTube]   \n",
       "2  Hotelplease                                      [Google Maps]   \n",
       "3    Donations                                           [PayPal]   \n",
       "4  PetStew.com  [Flickr, Amazon Product Advertising, Google Ma...   \n",
       "\n",
       "                                          Categories  \\\n",
       "0                    Mapping, Social, Visualizations   \n",
       "1                                              Video   \n",
       "2                                    Mapping, Travel   \n",
       "3                           Social, Charity, Charity   \n",
       "4  Pets, Pets, Classifieds, Video, Photos, eCommerce   \n",
       "\n",
       "                                                 URL Company Mashup/App Type  \\\n",
       "0                       http://twitwhere.appspot.com    None            None   \n",
       "1                              http://infinitube.net    None            None   \n",
       "2  http://hosting.gmodules.com/ig/gadgets/file/11...    None            None   \n",
       "3                    http://www.qmpeople.com/helpme/    None            None   \n",
       "4                  http://www.petstew.com/index.html    None            None   \n",
       "\n",
       "                                         Description  \n",
       "0  This Service is a mashup service using twitter...  \n",
       "1  Youtube-powered infinitely-playing video viewe...  \n",
       "2  Search the prices on 43,000 hotels worldwide. ...  \n",
       "3  Thanks to PayPal, users can donate money to he...  \n",
       "4  A pet site providing classifieds, adoptions, p...  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mashup.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "f11b0dc7",
   "metadata": {},
   "outputs": [],
   "source": [
    "mashup_non_trivial = mashup[mashup['Related APIs'].map(len) > 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "f283a808",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mashup_name</th>\n",
       "      <th>Related APIs</th>\n",
       "      <th>Categories</th>\n",
       "      <th>URL</th>\n",
       "      <th>Company</th>\n",
       "      <th>Mashup/App Type</th>\n",
       "      <th>Description</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Twitwhere</td>\n",
       "      <td>[Twitter, Google Earth, Google App Engine]</td>\n",
       "      <td>Mapping, Social, Visualizations</td>\n",
       "      <td>http://twitwhere.appspot.com</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>This Service is a mashup service using twitter...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>PetStew.com</td>\n",
       "      <td>[Flickr, Amazon Product Advertising, Google Ma...</td>\n",
       "      <td>Pets, Pets, Classifieds, Video, Photos, eCommerce</td>\n",
       "      <td>http://www.petstew.com/index.html</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>A pet site providing classifieds, adoptions, p...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Audio Transcription</td>\n",
       "      <td>[US Yellow Pages, Yahoo My Web Search]</td>\n",
       "      <td>Transcription, Accessibility</td>\n",
       "      <td>http://transcriptionplace.com/audio-book-trans...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Transcription supports accessibility. Persons ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>CityClash</td>\n",
       "      <td>[Google Maps, Google Cloud Translation, Google...</td>\n",
       "      <td>Social, Ratings, Ratings, Travel, Real Estate,...</td>\n",
       "      <td>http://www.cityclash.org/</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>City ratings and rankings by people.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Moveable History</td>\n",
       "      <td>[Wikipedia, Tropo Scripting, Voxeo, Google Lat...</td>\n",
       "      <td>Mobile, Voice, Mapping, History</td>\n",
       "      <td>http://moveable-weather.appspot.com/history_demo</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>This is the 3rd in the \"moveable\" series, that...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           mashup_name                                       Related APIs  \\\n",
       "0            Twitwhere         [Twitter, Google Earth, Google App Engine]   \n",
       "4          PetStew.com  [Flickr, Amazon Product Advertising, Google Ma...   \n",
       "5  Audio Transcription             [US Yellow Pages, Yahoo My Web Search]   \n",
       "6            CityClash  [Google Maps, Google Cloud Translation, Google...   \n",
       "7     Moveable History  [Wikipedia, Tropo Scripting, Voxeo, Google Lat...   \n",
       "\n",
       "                                          Categories  \\\n",
       "0                    Mapping, Social, Visualizations   \n",
       "4  Pets, Pets, Classifieds, Video, Photos, eCommerce   \n",
       "5                       Transcription, Accessibility   \n",
       "6  Social, Ratings, Ratings, Travel, Real Estate,...   \n",
       "7                    Mobile, Voice, Mapping, History   \n",
       "\n",
       "                                                 URL Company Mashup/App Type  \\\n",
       "0                       http://twitwhere.appspot.com    None            None   \n",
       "4                  http://www.petstew.com/index.html    None            None   \n",
       "5  http://transcriptionplace.com/audio-book-trans...    None            None   \n",
       "6                          http://www.cityclash.org/    None            None   \n",
       "7   http://moveable-weather.appspot.com/history_demo    None            None   \n",
       "\n",
       "                                         Description  \n",
       "0  This Service is a mashup service using twitter...  \n",
       "4  A pet site providing classifieds, adoptions, p...  \n",
       "5  Transcription supports accessibility. Persons ...  \n",
       "6               City ratings and rankings by people.  \n",
       "7  This is the 3rd in the \"moveable\" series, that...  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mashup_non_trivial.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "4e723361",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'This is the 3rd in the \"moveable\" series, that started with Moveable Weather, and then moved on to Moveable Tides. This time, this Tropo/Google Latitude/Wikipedia mashup reads to from Wikipedia pages about places registered on the National Registry of Historic Places. (U.S only). You can surf virtually, or actually visit the locations with your GPS-enabled phone.'"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mashup_non_trivial.iloc[4]['Description']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "3eea2248",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sequence = 'A pet site providing classifieds, adoptions, photos, videos, pricing info, breeder directories, shopping, breed information, and other useful tools for a healthy and happy relationship with your pet.'\n",
    "model(sequence)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "2101ce57",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokens = nltk.word_tokenize(sequence)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "0c6a8083",
   "metadata": {},
   "outputs": [],
   "source": [
    "# nltk.set_proxy('http://127.0.0.1:18080')\n",
    "# nltk.download('averaged_perceptron_tagger')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "f0ce8423",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('A', 'DT'),\n",
       " ('pet', 'JJ'),\n",
       " ('site', 'NN'),\n",
       " ('providing', 'VBG'),\n",
       " ('classifieds', 'NNS'),\n",
       " (',', ','),\n",
       " ('adoptions', 'NNS'),\n",
       " (',', ','),\n",
       " ('photos', 'NNS'),\n",
       " (',', ','),\n",
       " ('videos', 'NNS'),\n",
       " (',', ','),\n",
       " ('pricing', 'VBG'),\n",
       " ('info', 'NN'),\n",
       " (',', ','),\n",
       " ('breeder', 'NN'),\n",
       " ('directories', 'NNS'),\n",
       " (',', ','),\n",
       " ('shopping', 'NN'),\n",
       " (',', ','),\n",
       " ('breed', 'NN'),\n",
       " ('information', 'NN'),\n",
       " (',', ','),\n",
       " ('and', 'CC'),\n",
       " ('other', 'JJ'),\n",
       " ('useful', 'JJ'),\n",
       " ('tools', 'NNS'),\n",
       " ('for', 'IN'),\n",
       " ('a', 'DT'),\n",
       " ('healthy', 'JJ'),\n",
       " ('and', 'CC'),\n",
       " ('happy', 'JJ'),\n",
       " ('relationship', 'NN'),\n",
       " ('with', 'IN'),\n",
       " ('your', 'PRP$'),\n",
       " ('pet', 'NN'),\n",
       " ('.', '.')]"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nltk.pos_tag(tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "57fa2b14",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'This Service is a mashup service using twitter and google maps apis. It displays a users twitter status on the globe.'"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mashup.iloc[0]['Description']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "b549e6c3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model(mashup.iloc[0]['Description'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4213474e",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
